
function [ results ] = clean( filename, isPreprocessed, gen_flag  )

load_settings;

% isPreprocessed flag just indicates whether unnecessary columns had been
% deleted. This should be true ONLY for the original data (te very first
% run). All new mturk data should set this to false.

% gen_flag specifies whether random negative examples should be generated.
% This should be set to false for iter_data, true for all other sets.

%% Results struct type
field1 = 'cam_id'; value1 = 0;
field2 = 'day'; value2= 0;
field3 = 'hour'; value3 = 0;
field4 = 'ppl_rects'; value4 = [];
field5 = 'ppl_hogs'; value5 = [];
field6 = 'bg_rects'; value6 = [];
field7 = 'bg_hogs'; value7 = [];
field8 = 'set'; value8 = false;
field9 = 'hog_dims'; value9 = [];
field10 = 'filename'; value10 = '';

results = struct(field1, value1, field2, value2, field3, value3, field4, value4, field5, value5, field6, value6, field7, value7, field8, value8, field9, value9, field10, value10);


%%
imported = importdata(filename);
disp('Clean: Importing data');
for ix = 1:length(imported)
    imported(ix) = regexprep(imported(ix), '"', ''); % Remove quotes
    data(ix) = regexp(imported(ix), '\t', 'split'); % tab delimited file
    cur_img = data{ix};
    if ~isPreprocessed
        cur_img(cellfun('isempty',cur_img)) = []; % delete blank cells (usually caused by extra tabs or something)
    end
    data{ix} = cur_img; % save it back
end
clear cur_img;
clear imported;

%% MTurk specific stuff
% delete unneccessary columns and headers

%         cur_data{31:32} = []; % delete Submit box and noPeople
%         cur_data{15:29} = []; % delete assignmentduration, autoapprovaldelay, hitlifetime, viewhit, assignmentid, workerid, assignmentstatus, autoapprovaltime, assignmentaccepttime, assignmentsubmittime, assignmentapprovaltime, assignmentrejecttime, deadline, feedback, reject columns
%         cur_data{1:13} = [];  % delete hitid, hittypeid, title, description, keywords, reward, creationtime, assignments, numavailable, numpending, numcomplete, hitstatus, reviewstats columns
if ~isPreprocessed
    for ix = 2:length(data)
        cur_data = data{ix};
        for jx = [ 24 23 22 21 20 19 18 17 16 15 13 12 11 10 9 8 7 6 5 4 3 2 1]
            cur_data{jx} = [];
        end
        cur_data(cellfun('isempty',cur_data)) = []; % delete blank cells (usually caused by extra tabs or something)
        for jx = 1:length(cur_data)
            if strcmp(cur_data{jx}, 'noPeople') || strcmp(cur_data{jx}, 'Submit')
                cur_data{jx} = [];
            end
        end
        cur_data(cellfun('isempty',cur_data)) = [];
        if length(cur_data) > 1 % have rectangles!
            data{ix} = cur_data;
        else
            data{ix} = [];
        end
    end
    data{1} = []; % delete headers
    data(cellfun('isempty',data)) = [];
    % move data around and parse
    for ix = 1:length(data)
        cur_data = data{ix};
        name = regexp(cur_data{1}, '/', 'split');
        temp = cell(1);
        temp{1} = cur_data{1};
        temp{2} = name{5};
        temp{3} = name{6};
        for jx = 2:length(cur_data)
            temp{2+jx} = cur_data{jx};
        end
        data{ix} = temp;
        clear temp;
        clear name;
    end
end

clear cur_data;

%% Remove blanks all over the place
data(cellfun('isempty',data)) = [];
for ix = 1:length(data)
    current = data{ix};
    current(cellfun('isempty',current)) = [];    
    for jx = 1:length(current)
        if strcmp(current{jx}, '')
            current{jx} = [];
        end
    end
    data{ix} = current;
end


%% Move data around

for ix = 1:length(data)
    current = data{ix};
    ll = length(current);
    for jx = 0:ll-4 % make room for extra values
        current{ll+1-jx} = current{ll-jx};
    end
    current{4} = current{3};
    cam_id = current{2};
    current{1} = str2double(cam_id);
    split = regexp(current{3}, '_', 'split');
    day = str2double(split{1});
    hour = split{2};
    hour = str2double(hour(1:end-4));
    current{2} = day;
    current{3} = hour;
    data{ix} = current;
end


%%

% Append boxes -> DEPENDS ON IMPORTED DATA BEING ORDERED BY ANNOTATION
disp('Clean: Appending boxes');
img_index = 0;
prev_cam_id = 0;
prev_day = 0;
prev_hour = 0;
for ix = 1:length(data)
    current = data{ix};
    if (prev_cam_id ~= current{1}) || (prev_day ~= current{2}) || (prev_hour ~= current{3}) % new image
        img_index = img_index+1;
        results(img_index).cam_id = current{1};
        results(img_index).day = current{2};
        results(img_index).hour = current{3};
        results(img_index).filename = current{4};
        prev_cam_id = current{1};
        prev_day = current{2};
        prev_hour = current{3};
    end

    
    for jx = 5:length(current)
        nums = regexp(current{jx}, ', ', 'split');
        % If any of the vertices are undefined
        if ~strcmp(nums{1}, 'undefined');
            if ~strcmp(nums{2}, 'undefined');
                if ~strcmp(nums{3}, 'undefined');
                    if ~strcmp(nums{4}, 'undefined');
                        % add to results
                        cur_rect = zeros(1,4);
                        for tx = 1:4
                            cur_rect(1, tx) = int64(str2double(nums{tx}));
                        end
                        results(img_index).ppl_rects = [ results(img_index).ppl_rects; cur_rect];
                    end
                end
            end
        end
    end
end
%%
clear data;


%% Cleaning
results = checkRects(results, gen_flag);

disp('Clean: Done cleaning.');
end